##### import packages
#base
import os
import sys
from collections import defaultdict
import numpy as np
import scipy.stats
from matplotlib import pyplot as plt
import random
# import pyreadr
import pandas as pd
import seaborn as sns
%matplotlib inline
%autosave 30
#misc
import pickle
# import torch.nn.functional as F
import itertools
import time
import umap
import plotly
import plotly.graph_objs as go
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
# import homebrew modules
import models.tomtom_models as tm
import models.tomtom_util as tu
global tself_norm_all_3d, tself_norm_noauto_3d, tself_raw_all_3d, tself_raw_noauto_3d
global ttarg_norm_all_3d, ttarg_norm_noauto_3d, ttarg_raw_all_3d, ttarg_raw_noauto_3d
global tavg_norm_all_3d, tavg_norm_noauto_3d, tavg_raw_all_3d, tavg_raw_noauto_3d
# import pickled data
with open('data/tomtom_data_preprocessed.pkl','rb') as f:
[tself_norm_all_3d, tself_norm_noauto_3d, tself_raw_all_3d, tself_raw_noauto_3d,
ttarg_norm_all_3d, ttarg_norm_noauto_3d, ttarg_raw_all_3d, ttarg_raw_noauto_3d,
tavg_norm_all_3d, tavg_norm_noauto_3d, tavg_raw_all_3d, tavg_raw_noauto_3d] = pickle.load(f)
# import fitted models
with open('tomtom_fitted_models.pkl','rb') as f:
[seeds_self_norm_all_grp,maps_self_norm_all_grp,logprobs_self_norm_all_grp,mems_self_norm_all_grp,
seeds_self_norm_all_dim,maps_self_norm_all_dim,logprobs_self_norm_all_dim,
seeds_self_norm_noauto_grp,maps_self_norm_noauto_grp,logprobs_self_norm_noauto_grp,mems_self_norm_noauto_grp,
seeds_self_norm_noauto_dim,maps_self_norm_noauto_dim,logprobs_self_norm_noauto_dim,
seeds_self_raw_all_grp,maps_self_raw_all_grp,logprobs_self_raw_all_grp,mems_self_raw_all_grp,
seeds_self_raw_all_dim,maps_self_raw_all_dim,logprobs_self_raw_all_dim,
seeds_self_raw_noauto_grp,maps_self_raw_noauto_grp,logprobs_self_raw_noauto_grp,mems_self_raw_noauto_grp,
seeds_self_raw_noauto_dim,maps_self_raw_noauto_dim,logprobs_self_raw_noauto_dim] = pickle.load(f)
def pca_scree(data):
scaler = StandardScaler()
data = data.reshape(data.shape[0],-1)
dt_scaled = scaler.fit_transform(data)
decomp = PCA()
dtdecomp = decomp.fit(dt_scaled)
plt.scatter(list(range(np.shape(dtdecomp.explained_variance_ratio_)[0])),np.array(dtdecomp.explained_variance_ratio_))
def viz_umap_2d(data,mem,n_nbr = 50,min_dist = .1,random_state = 9):
random.seed(random_state)
mapper = umap.UMAP(n_neighbors = n_nbr, min_dist = min_dist)
data = data.reshape(data.shape[0],-1)
udt = mapper.fit_transform(data)
sns.scatterplot(udt[:,0],udt[:,1],hue = mem)
def viz_umap_3d(data,mem,n_nbr = 50,min_dist = .1,random_state = 9):
random.seed(random_state)
mapper = umap.UMAP(n_neighbors = n_nbr, min_dist = min_dist, n_components = 3)
data = data.reshape(data.shape[0],-1)
udt = mapper.fit_transform(data)
# Configure Plotly to be rendered inline in the notebook.
plotly.offline.init_notebook_mode()
trace = go.Scatter3d(
x=udt[:,0],
y=udt[:,1],
z=udt[:,2],
mode='markers',
marker={
'size': 10,
'opacity': 0.8,
'color': mem,
}
)
# Configure the layout.
layout = go.Layout(
margin={'l': 0, 'r': 0, 'b': 0, 't': 0}
)
plot_figure = go.Figure(data=[trace], layout=layout)
# Render the plot.
plotly.offline.iplot(plot_figure)
pca_scree(tself_raw_noauto_3d)
viz_umap_2d(tself_raw_noauto_3d,mems_self_raw_noauto_grp[1])
viz_umap_3d(tself_raw_noauto_3d,mems_self_raw_noauto_grp[1])
viz_umap_2d(tself_raw_noauto_3d,mems_self_raw_noauto_grp[5])
viz_umap_3d(tself_raw_noauto_3d,mems_self_raw_noauto_grp[5])